{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "initial_id",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-21T12:36:25.657139Z",
     "start_time": "2024-10-21T12:36:25.108068Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'icourse163_page.html'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import requests\n",
    "\n",
    "# 目标网址\n",
    "url = \"https://www.icourse163.org/course/NHDX-1463126169\"\n",
    "\n",
    "# 设置请求头，模拟浏览器访问\n",
    "headers = {\n",
    "    \"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36\"\n",
    "}\n",
    "\n",
    "# 发起请求\n",
    "response = requests.get(url, headers=headers)\n",
    "\n",
    "# 将响应的内容保存为HTML文件\n",
    "file_path = 'icourse163_page.html'\n",
    "with open(file_path, 'w', encoding='utf-8') as file:\n",
    "    file.write(response.text)\n",
    "\n",
    "file_path\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "d636d4a6dbabd98b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-09-23T16:43:47.287593Z",
     "start_time": "2024-09-23T16:43:35.960573Z"
    }
   },
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "from time import sleep\n",
    "\n",
    "# 使用 Selenium Manager 自动管理 ChromeDriver\n",
    "driver = webdriver.Chrome()\n",
    "# driver = webdriver.Firefox()\n",
    "\n",
    "# 打开目标网页\n",
    "driver.get(\"https://www.icourse163.org/course/NHDX-1463126169\")\n",
    "\n",
    "# 等待页面加载\n",
    "sleep(5)\n",
    "\n",
    "# 保存网页源代码\n",
    "with open(\"icourse163_selenium_saved.html\", \"w\", encoding=\"utf-8\") as file:\n",
    "    file.write(driver.page_source)\n",
    "    \n",
    "\n",
    "driver.quit()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "63dcb77b3103a033",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-09-23T16:43:47.368352Z",
     "start_time": "2024-09-23T16:43:47.366041Z"
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "d89648e8-324a-4c91-b0af-541ab0276918",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-09-23T16:43:58.105880Z",
     "start_time": "2024-09-23T16:43:47.377122Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "点击了'课程评价'按钮\n",
      "页面已保存为 course_reviews.html\n",
      "浏览器已关闭\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "\n",
    "# 初始化 WebDriver，启动 Chrome 浏览器\n",
    "driver = webdriver.Chrome()\n",
    "\n",
    "# 打开目标网页\n",
    "url = \"https://www.icourse163.org/course/NHDX-1463126169\"\n",
    "driver.get(url)\n",
    "\n",
    "# 设置浏览器窗口大小\n",
    "driver.set_window_size(1082, 672)\n",
    "\n",
    "# 等待“课程评价”按钮加载，并点击\n",
    "try:\n",
    "    # 等待按钮可点击\n",
    "    review_button = WebDriverWait(driver, 10).until(\n",
    "        EC.element_to_be_clickable((By.ID, \"review-tag-button\"))\n",
    "    )\n",
    "    review_button.click()  # 点击“课程评价”按钮\n",
    "    print(\"点击了'课程评价'按钮\")\n",
    "except Exception as e:\n",
    "    print(f\"未能找到或点击按钮，错误: {e}\")\n",
    "\n",
    "# 保存当前页面的 HTML 源代码\n",
    "time.sleep(5)  # 等待页面完全加载\n",
    "file_name = \"course_reviews.html\"\n",
    "with open(file_name, \"w\", encoding=\"utf-8\") as file:\n",
    "    file.write(driver.page_source)  # 将页面源码写入文件\n",
    "print(f\"页面已保存为 {file_name}\")\n",
    "\n",
    "# 关闭浏览器\n",
    "driver.quit()\n",
    "print(\"浏览器已关闭\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "82ff7190-0660-4443-a293-059c91dfcc9d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-09-23T16:44:28.604274Z",
     "start_time": "2024-09-23T16:43:58.116754Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "正在提取第 1 页的评论...\n",
      "正在提取第 2 页的评论...\n",
      "正在提取第 3 页的评论...\n",
      "正在提取第 4 页的评论...\n",
      "正在提取第 5 页的评论...\n",
      "数据已保存到 course_reviews1.csv\n",
      "所有页面处理完毕，浏览器已关闭\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "import pandas as pd\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "\n",
    "# 创建一个 Chrome 浏览器实例\n",
    "driver = webdriver.Chrome()\n",
    "\n",
    "\n",
    "# 用于存储所有评论数据的列表\n",
    "all_reviews = []\n",
    "\n",
    "# 打开指定的网页\n",
    "url = \"https://www.icourse163.org/course/NHDX-1463126169\"\n",
    "driver.get(url)\n",
    "\n",
    "# 设置浏览器窗口大小\n",
    "driver.set_window_size(1082, 672)\n",
    "\n",
    "# 点击“课程评价”按钮，等待按钮可点击\n",
    "try:\n",
    "    review_button = WebDriverWait(driver, 10).until(\n",
    "        EC.element_to_be_clickable((By.ID, \"review-tag-button\"))\n",
    "    )\n",
    "    review_button.click()  # 点击按钮\n",
    "    time.sleep(3)  # 等待页面加载\n",
    "except Exception as e:\n",
    "    print(f\"无法点击课程评价按钮: {e}\")\n",
    "\n",
    "# 定义一个函数用于提取页面上的评论内容\n",
    "def extract_reviews():\n",
    "    # 查找页面上的所有评论元素\n",
    "    review_elements = driver.find_elements(By.XPATH, '//div[@class=\"ux-mooc-comment-course-comment_comment-list_item\"]')\n",
    "    \n",
    "    # 遍历每个评论元素\n",
    "    for review_element in review_elements:\n",
    "        review_data = {}\n",
    "        try:\n",
    "            # 提取用户名称\n",
    "            review_data['user'] = review_element.find_element(\n",
    "                By.CLASS_NAME, 'ux-mooc-comment-course-comment_comment-list_item_body_user-info_name'\n",
    "            ).text\n",
    "\n",
    "            # 提取评论时间\n",
    "            review_data['time'] = review_element.find_element(\n",
    "                By.CLASS_NAME, 'ux-mooc-comment-course-comment_comment-list_item_body_comment-info_time'\n",
    "            ).text\n",
    "\n",
    "            # 提取评分（星星数）\n",
    "            stars = review_element.find_elements(\n",
    "                By.CLASS_NAME, 'ux-icon-custom-rating-favorite'\n",
    "            )\n",
    "            review_data['rating'] = len(stars)  # 星星数表示评分\n",
    "\n",
    "            # 提取评论内容\n",
    "            review_data['content'] = review_element.find_element(\n",
    "                By.CLASS_NAME, 'ux-mooc-comment-course-comment_comment-list_item_body_content'\n",
    "            ).text\n",
    "        except Exception as e:\n",
    "            print(f\"解析评论时出错: {e}\")\n",
    "\n",
    "        # 将每个评论数据添加到总列表中\n",
    "        all_reviews.append(review_data)\n",
    "\n",
    "# 定义一个函数用于点击下一页按钮\n",
    "def next_page():\n",
    "    try:\n",
    "        # 查找并点击下一页按钮\n",
    "        next_button = WebDriverWait(driver, 10).until(\n",
    "            EC.element_to_be_clickable((By.CSS_SELECTOR, '#comment-section > div > div.ux-mooc-comment-course-comment_pager > ul > li.ux-pager_btn.ux-pager_btn__next > a'))\n",
    "        )\n",
    "        next_button.click()  # 点击下一页按钮\n",
    "        time.sleep(3)  # 等待页面加载\n",
    "        return True  # 成功点击下一页按钮\n",
    "    except Exception as e:\n",
    "        print(f\"无法点击下一页: {e}\")\n",
    "        return False  # 无法点击下一页按钮\n",
    "\n",
    "# 提取指定页数的评论\n",
    "num_pages = 5  # 设置要提取的页数\n",
    "for page in range(1, num_pages + 1):\n",
    "    print(f\"正在提取第 {page} 页的评论...\")\n",
    "    extract_reviews()  # 提取当前页面的评论\n",
    "\n",
    "    # 如果不是最后一页，尝试翻页\n",
    "    if not next_page():\n",
    "        print(\"已到达最后一页，停止翻页。\")\n",
    "        break\n",
    "\n",
    "# 将所有提取的评论数据保存为 CSV 文件\n",
    "df = pd.DataFrame(all_reviews)\n",
    "output_file = \"course_reviews1.csv\"\n",
    "df.to_csv(output_file, index=False, encoding='utf-8-sig')\n",
    "print(f\"数据已保存到 {output_file}\")\n",
    "\n",
    "# 关闭浏览器\n",
    "driver.quit()\n",
    "print(\"所有页面处理完毕，浏览器已关闭\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e338421d-74f8-4526-b713-bf4839f74265",
   "metadata": {},
   "source": [
    "如果你浏览器有问题 可以用其他浏览器代替\n",
    "self.driver = webdriver.Firefox()\n",
    "将 Chrome WebDriver 换成 Firefox WebDriver。\n",
    "。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "673105c3-4c4f-40d1-89b7-e7be00d026fe",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "正在提取第 1 页的评论...\n",
      "正在提取第 2 页的评论...\n",
      "正在提取第 3 页的评论...\n",
      "正在提取第 4 页的评论...\n",
      "正在提取第 5 页的评论...\n",
      "数据已保存到 course_reviewsfox.csv\n",
      "所有页面处理完毕，浏览器已关闭\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "import pandas as pd\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "\n",
    "# 创建一个 Firefox 浏览器实例\n",
    "driver = webdriver.Firefox()  # 使用Firefox代替Chrome\n",
    "\n",
    "# 用于存储所有评论数据的列表\n",
    "all_reviews = []\n",
    "\n",
    "# 打开指定的网页\n",
    "url = \"https://www.icourse163.org/course/NHDX-1463126169\"\n",
    "driver.get(url)\n",
    "\n",
    "# 设置浏览器窗口大小\n",
    "driver.set_window_size(1082, 672)\n",
    "\n",
    "# 点击“课程评价”按钮，等待按钮可点击\n",
    "try:\n",
    "    review_button = WebDriverWait(driver, 10).until(\n",
    "        EC.element_to_be_clickable((By.ID, \"review-tag-button\"))\n",
    "    )\n",
    "    review_button.click()  # 点击按钮\n",
    "    time.sleep(3)  # 等待页面加载\n",
    "except Exception as e:\n",
    "    print(f\"无法点击课程评价按钮: {e}\")\n",
    "\n",
    "# 定义一个函数用于提取页面上的评论内容\n",
    "def extract_reviews():\n",
    "    # 查找页面上的所有评论元素\n",
    "    review_elements = driver.find_elements(By.XPATH, '//div[@class=\"ux-mooc-comment-course-comment_comment-list_item\"]')\n",
    "    \n",
    "    # 遍历每个评论元素\n",
    "    for review_element in review_elements:\n",
    "        review_data = {}\n",
    "        try:\n",
    "            # 提取用户名称\n",
    "            review_data['user'] = review_element.find_element(\n",
    "                By.CLASS_NAME, 'ux-mooc-comment-course-comment_comment-list_item_body_user-info_name'\n",
    "            ).text\n",
    "\n",
    "            # 提取评论时间\n",
    "            review_data['time'] = review_element.find_element(\n",
    "                By.CLASS_NAME, 'ux-mooc-comment-course-comment_comment-list_item_body_comment-info_time'\n",
    "            ).text\n",
    "\n",
    "            # 提取评分（星星数）\n",
    "            stars = review_element.find_elements(\n",
    "                By.CLASS_NAME, 'ux-icon-custom-rating-favorite'\n",
    "            )\n",
    "            review_data['rating'] = len(stars)  # 星星数表示评分\n",
    "\n",
    "            # 提取评论内容\n",
    "            review_data['content'] = review_element.find_element(\n",
    "                By.CLASS_NAME, 'ux-mooc-comment-course-comment_comment-list_item_body_content'\n",
    "            ).text\n",
    "        except Exception as e:\n",
    "            print(f\"解析评论时出错: {e}\")\n",
    "\n",
    "        # 将每个评论数据添加到总列表中\n",
    "        all_reviews.append(review_data)\n",
    "\n",
    "# 定义一个函数用于点击下一页按钮\n",
    "def next_page():\n",
    "    try:\n",
    "        # 查找并点击下一页按钮\n",
    "        next_button = WebDriverWait(driver, 10).until(\n",
    "            EC.element_to_be_clickable((By.CSS_SELECTOR, '#comment-section > div > div.ux-mooc-comment-course-comment_pager > ul > li.ux-pager_btn.ux-pager_btn__next > a'))\n",
    "        )\n",
    "        next_button.click()  # 点击下一页按钮\n",
    "        time.sleep(3)  # 等待页面加载\n",
    "        return True  # 成功点击下一页按钮\n",
    "    except Exception as e:\n",
    "        print(f\"无法点击下一页: {e}\")\n",
    "        return False  # 无法点击下一页按钮\n",
    "\n",
    "# 提取指定页数的评论\n",
    "num_pages = 5  # 设置要提取的页数\n",
    "for page in range(1, num_pages + 1):\n",
    "    print(f\"正在提取第 {page} 页的评论...\")\n",
    "    extract_reviews()  # 提取当前页面的评论\n",
    "\n",
    "    # 如果不是最后一页，尝试翻页\n",
    "    if not next_page():\n",
    "        print(\"已到达最后一页，停止翻页。\")\n",
    "        break\n",
    "\n",
    "# 将所有提取的评论数据保存为 CSV 文件\n",
    "df = pd.DataFrame(all_reviews)\n",
    "output_file = \"course_reviewsfox.csv\"\n",
    "df.to_csv(output_file, index=False, encoding='utf-8-sig')\n",
    "print(f\"数据已保存到 {output_file}\")\n",
    "\n",
    "# 关闭浏览器\n",
    "driver.quit()\n",
    "print(\"所有页面处理完毕，浏览器已关闭\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "562aab8b-4fc5-4607-90e7-f31bd97ecd7e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
